/**
 * Copyright 2019-2020 Huawei Technologies Co., Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "ops_kernel_store/fe_ops_kernel_info_store.h"
#include <utils/tensor_utils.h>
#include <algorithm>
#include <climits>
#include <cstdlib>
#include <fstream>
#include <sstream>
#include <utility>
#include <vector>
#include "common/configuration.h"
#include "common/fe_error_code.h"
#include "common/fe_inner_attr_define.h"
#include "common/fe_inner_error_codes.h"
#include "common/fe_utils.h"
#include "common/unknown_shape_util.h"
#include "common/util/op_info_util.h"
#include "graph/debug/ge_attr_define.h"
#include "graph/ge_tensor.h"
#include "graph/op_kernel_bin.h"
#include "graph_optimizer/op_compiler/tbe_json_parse.h"
#include "ops_store/ops_kernel_manager.h"
#include "param_calculate/tensorsize_calculator.h"

using std::map;
using std::string;
using std::vector;
namespace fe {
using fe::StringUtils;
using TbeJsonFileParsePtr = std::shared_ptr<TbeJsonFileParse>;
using ScopeJsonMap_t = std::map<int64_t, std::string>;
/*
 *  @ingroup fe
 *  @brief   check specified GeAttrValue::ValueType of op_desc_attr.Value() is in
 * info_op_attr.vector<Values>
 *  @param   [in] value      : used to specified GeAttrValue::ValueType for
 * template
 *  @param   [in] op_desc_attr : GeAttrValue from OpDesc
 *  @param   [in] info_op_attr : vector<GeAttrValue> from OpKernelInfo
 *  @return  true or false
 */
template <typename T>
bool FindValueInVector(T &value, const ge::GeAttrValue &op_desc_attr, const vector<ge::GeAttrValue> &info_op_attr);

FEOpsKernelInfoStore::FEOpsKernelInfoStore(OpStoreAdapterManagerPtr op_store_adapter_manager_ptr,
                                           std::string engine_name)
    : init_flag_(false),
      custom_or_builtin_exist_flag_(false),
      map_all_sub_store_info_(),
      op_kernel_store_type_(),
      engine_name_(engine_name),
      op_store_adapter_manager_ptr_(op_store_adapter_manager_ptr),
      check_support_static_flag_(false) {}

FEOpsKernelInfoStore::~FEOpsKernelInfoStore() {}

Status FEOpsKernelInfoStore::Initialize(const map<string, string> &options) {
  if (init_flag_) {
    FE_LOGD("FEOpsKernelStore has already been initialized.");
    return SUCCESS;
  }
  FE_CHECK(op_store_adapter_manager_ptr_ == nullptr,
           REPORT_FE_ERROR("[GraphOpt][SetCheck][Init] op_store_adapter_manager_ptr_ is null."), return FAILED);
  /* Before FEOpsKernelInfoStore is initialized, Configuration class has
     already loaded ops info lib info vector */
  init_flag_ = true;
  Status status = OpsKernelManager::Instance(engine_name_).Initialize();
  if (status != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][SetCheck][Init] Fail to initialize ops kernel manager.");
    return FAILED;
  }

  const std::vector<FEOpsStoreInfo> ops_store_info_vec = Configuration::Instance(engine_name_).GetOpsStoreInfo();
  for (const FEOpsStoreInfo &ops_sub_store_info : ops_store_info_vec) {
    SubOpsStorePtr sub_ops_kernel_info_store_ptr = nullptr;
    FE_MAKE_SHARED(sub_ops_kernel_info_store_ptr = std::make_shared<SubOpsStore>(op_store_adapter_manager_ptr_),
                   return OP_STORE_MAKE_SHARED_FAILED);
    FE_CHECK(sub_ops_kernel_info_store_ptr == nullptr,
             REPORT_FE_ERROR("[GraphOpt][SetCheck][Init] subOpsKernelInfoStorePtr is nullptr."),
             return PARAM_INVALID);
    sub_ops_kernel_info_store_ptr->SetSubStoreInfo(ops_sub_store_info);
    Status result = sub_ops_kernel_info_store_ptr->InitializeSubStore(engine_name_);
    if (result == SUCCESS) {
      map_all_sub_store_info_.emplace(
          std::make_pair(ops_sub_store_info.fe_ops_store_name, sub_ops_kernel_info_store_ptr));
    }
  }

  return SUCCESS;
}

Status FEOpsKernelInfoStore::Finalize() {
  FE_LOGD("Finalizing the FEOpsKernelStore...");
  if (!init_flag_) {
    FE_LOGD("FEOpsKernelInfoStore has not been initialized, Finalize is not allowed.");
    return SUCCESS;
  }
  Status status = OpsKernelManager::Instance(engine_name_).Finalize();
  if (status != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][SetCheck][Finalize] Fail to finalize ops kernel manager.");
    return FAILED;
  }
  init_flag_ = false;
  return SUCCESS;
}

Status FEOpsKernelInfoStore::CreateSession(const std::map<std::string, std::string> &session_options) {
  return SUCCESS;
}

Status FEOpsKernelInfoStore::DestroySession(const std::map<std::string, std::string> &session_options) {
  return SUCCESS;
}

std::string FEOpsKernelInfoStore::GetFEOpsKernelInfoStoreName() const { return engine_name_; }

void FEOpsKernelInfoStore::GetAllOpsKernelInfo(map<string, ge::OpInfo> &infos) const {
  OpsKernelManager::Instance(engine_name_).GetAllOpsKernelInfo(infos);
}

Status FEOpsKernelInfoStore::GetTbeAdapter(const ge::OpDescPtr &op_desc_ptr, const FEOpsStoreInfo &ops_store,
                                           std::ostringstream &reason_oss, OpStoreAdapterPtr &op_store_adapter,
                                           SubOpsStorePtr &sub_ops_store) const {
  FE_CHECK(op_store_adapter_manager_ptr_ == nullptr,
           REPORT_FE_ERROR("[GraphOpt][SetCheck][GetTbeAdap] op_store_adapter_manager_ptr_ is null."), return FAILED);
  if (op_store_adapter_manager_ptr_->GetOpStoreAdapter(ops_store.op_impl_type, op_store_adapter) != SUCCESS) {
    reason_oss << "The reason why this op " << op_desc_ptr->GetName() << " is not supported by op information library["
               << ops_store.fe_ops_store_name << "] is that get op information library adapter failed." << endl;
    return FAILED;
  }

  auto sub_ops_store_iter = map_all_sub_store_info_.find(ops_store.fe_ops_store_name);
  if (sub_ops_store_iter == map_all_sub_store_info_.end()) {
    reason_oss << "The reason why this op " << op_desc_ptr->GetName() << " is not supported by op information library["
               << ops_store.fe_ops_store_name << "] is that this sub ops information library not found." << endl;
    return FAILED;
  } else if (sub_ops_store_iter->second == nullptr) {
    reason_oss << "The reason why this op " << op_desc_ptr->GetName() << " is not supported by op information library["
               << ops_store.fe_ops_store_name
               << "] is that this sub ops information library is nullptr in map_all_sub_store_info_." << endl;
    return FAILED;
  }
  sub_ops_store = sub_ops_store_iter->second;
  return SUCCESS;
}

Status FEOpsKernelInfoStore::GetOpKernel(const ge::OpDescPtr &op_desc_ptr, const FEOpsStoreInfo &ops_store,
                                         OpKernelInfoPtr &op_kernel_ptr, std::ostringstream &reason_oss,
                                         uint64_t &not_support_reason_id) const {
  op_kernel_ptr = OpsKernelManager::Instance(engine_name_)
                      .GetOpKernelInfoByOpType(ops_store.fe_ops_store_name, op_desc_ptr->GetType());
  if (op_kernel_ptr == nullptr) {
    reason_oss << "The reason why this op " << op_desc_ptr->GetName() << " is not supported by op information library["
               << ops_store.fe_ops_store_name << "] is that "
               << "op type " << op_desc_ptr->GetType() << " is not found." << endl;
    uint64_t offset_num = ops_store.op_impl_type * NOT_SUPPORTED_REASON_OFFSET_BIT_NUM;
    not_support_reason_id += (static_cast<uint64_t>(EN_TYPE_NOT_FOUND) << offset_num);
    return FAILED;
  }
  return SUCCESS;
}

bool FEOpsKernelInfoStore::CheckSupportedByOpsStore(const ge::OpDescPtr &op_desc_ptr, UnSupportedReason &reason,
                                                    OpImplType &impl_type, CheckSupportMode mode) const {
  std::vector<FEOpsStoreInfo> fe_ops_store_info_vec = Configuration::Instance(engine_name_).GetOpsStoreInfo();

  std::ostringstream reason_oss;
  bool is_custom_op = false;
  size_t index = 0;
  for (auto &ops_store : fe_ops_store_info_vec) {
    UnSupportedReason sub_store_reason;
    index++;
    /*
     * non-persistent custom op,
     * only use non-persistent-tbe-custom op info lib and tbe-custom op info lib
     * other op can not use non-persistent-tbe-custom op info lib
     */
    bool ret = CheckCustomOp(op_desc_ptr, ops_store, is_custom_op);
    OpStoreAdapterPtr op_store_adapter = nullptr;
    SubOpsStorePtr sub_ops_store = nullptr;
    OpKernelInfoPtr op_kernel_ptr = nullptr;
    bool skip_check =
        ret || GetTbeAdapter(op_desc_ptr, ops_store, reason_oss, op_store_adapter, sub_ops_store) != SUCCESS ||
        GetOpKernel(op_desc_ptr, ops_store, op_kernel_ptr, reason_oss, sub_store_reason.reason_id) != SUCCESS;
    if (skip_check) {
      continue;
    }

    /* if curent op us unknown shape, we check ints unknown shape op info lib info
     * firstly, if is supported, set support_dynamicshape to "true", else, check
     * the normal info lib info, if is supported, set support_dynamicshape to
     * "false", else, return false. */
    bool fuzz_build = false;
    bool has_fuzz_build_attr = ge::AttrUtils::GetBool(op_desc_ptr, ge::ATTR_NAME_FUZZ_BUILD, fuzz_build) && fuzz_build;
    sub_store_reason.reason += "[Dynamic check]: ";
    bool flag_judge_dyn_spt =
        sub_ops_store->CheckSubStoreSupported(*(op_desc_ptr.get()), op_kernel_ptr, sub_store_reason, mode, true) &&
        OpCheckSupport(op_desc_ptr, ops_store, op_store_adapter, op_kernel_ptr, sub_store_reason);

    if (flag_judge_dyn_spt) {
      impl_type = ops_store.op_impl_type;
      FE_LOGD("Determine whether to support dyn shape, node name is %s.", op_desc_ptr->GetName().c_str());
      if (has_fuzz_build_attr) {
        if (op_kernel_ptr->GetSupportDynamicShape()) {
          (void)ge::AttrUtils::SetBool(op_desc_ptr, STR_SUPPORT_DYNAMIC_SHAPE, true);
          FE_LOGD("fuzz_build op:%s is set support dyn shape", op_desc_ptr->GetName().c_str());
        } else {
          FE_LOGD("fuzz_build op:%s use accurately build.", op_desc_ptr->GetName().c_str());
          (void)ge::AttrUtils::SetBool(op_desc_ptr, STR_FUZZ_BUILD_USE_ACCURATE, true);
        }
        SetGeOp(op_desc_ptr, op_kernel_ptr->GetOpInfo().opFileName);
        return true;
      } else {
        bool flag_support_dyn = op_kernel_ptr->GetSupportDynamicShape() &&
                                (!IsContainUnknownDimNum(*op_desc_ptr) || op_kernel_ptr->GetSupportDynamicRank()) &&
                                IsUnKnownShapeOp(*(op_desc_ptr.get()));
        if (flag_support_dyn) {
          (void)ge::AttrUtils::SetBool(op_desc_ptr, STR_SUPPORT_DYNAMIC_SHAPE, true);
          FE_LOGI("[ChkSpt][FEChk][Node %s, type %s] This op supports DYNAMIC shape.", op_desc_ptr->GetName().c_str(),
                  op_desc_ptr->GetType().c_str());
          SetGeOp(op_desc_ptr, op_kernel_ptr->GetOpInfo().opFileName);
          return true;
        } else {
          FE_LOGI("[ChkSpt][FEChk][Node %s, type %s] DYNAMIC shape check does not pass in op info lib %s.",
                  op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str(), ops_store.fe_ops_store_name.c_str());
        }
      }
    }

    sub_store_reason.reason += "[Static check]: ";
    if (sub_ops_store->CheckSubStoreSupported(*(op_desc_ptr.get()), op_kernel_ptr, sub_store_reason, mode, false)) {
      FE_LOGD("[ChkSpt][FEChk][Node %s, type %s] STATIC shape check passes in op info lib %s",
              op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str(), ops_store.fe_ops_store_name.c_str());
      impl_type = ops_store.op_impl_type;
      if (OpCheckSupport(op_desc_ptr, ops_store, op_store_adapter, op_kernel_ptr, sub_store_reason)) {
        FE_LOGD("Determine whether dyn shape are not supported, node name is %s.", op_desc_ptr->GetName().c_str());
        if (has_fuzz_build_attr) {
          (void)ge::AttrUtils::SetBool(op_desc_ptr, STR_FUZZ_BUILD_USE_ACCURATE, true);
          FE_LOGD("fuzz_build op:%s use accurately build.", op_desc_ptr->GetName().c_str());
        }
        if (IsUnKnownShapeOp(*(op_desc_ptr.get()))) {
          (void)ge::AttrUtils::SetBool(op_desc_ptr, STR_SUPPORT_DYNAMIC_SHAPE, false);
          FE_LOGD("Op %s, type %s supports DYNAMIC shape", op_desc_ptr->GetName().c_str(),
                  op_desc_ptr->GetType().c_str());
        }
        SetGeOp(op_desc_ptr, op_kernel_ptr->GetOpInfo().opFileName);
        return true;
      }
    }

    reason_oss << "The reason why this op " << op_desc_ptr->GetName() << " is not supported by op information library["
               << ops_store.fe_ops_store_name << "] is that " << sub_store_reason.reason;
    if (index != fe_ops_store_info_vec.size()) {
      reason_oss << endl;
    }
    uint64_t offset_num = ops_store.op_impl_type * NOT_SUPPORTED_REASON_OFFSET_BIT_NUM;
    FE_UINT64_ADDCHECK(reason.reason_id, (static_cast<uint64_t>(sub_store_reason.reason_id) << offset_num));
    reason.reason_id += (static_cast<uint64_t>(sub_store_reason.reason_id) << offset_num);
    reason.reason += reason_oss.str();
  }
  FE_LOGI("%s", reason_oss.str().c_str());

  return false;
}

Status FEOpsKernelInfoStore::GetNotSupportedReasonByAttr(uint64_t &reason, std::ostringstream &reason_oss) const {
  std::vector<FEOpsStoreInfo> fe_ops_store_info_vec = Configuration::Instance(engine_name_).GetOpsStoreInfo();
  for (auto &ops_store : fe_ops_store_info_vec) {
    FE_UINT32_MULCHECK(static_cast<uint32_t>(ops_store.op_impl_type), NOT_SUPPORTED_REASON_OFFSET_BIT_NUM);
    uint64_t offset_num = static_cast<uint64_t>(ops_store.op_impl_type * NOT_SUPPORTED_REASON_OFFSET_BIT_NUM);
    int64_t reason_id = static_cast<int64_t>((reason >> offset_num) & NOT_SUPPORTED_REASON_OFFSET_UNIT);
    OpNotSupportedReasonID sub_reason_id = (OpNotSupportedReasonID)reason_id;
    auto reason_inter = ID_REASON_MAP.find(sub_reason_id);
    if (reason_inter != ID_REASON_MAP.end()) {
      reason_oss << reason_inter->second << " Op information library name is " << ops_store.fe_ops_store_name << "."
                 << endl;
    } else {
      FE_LOGD("Get not supported reason in %s not success.", ops_store.fe_ops_store_name.c_str());
    }
  }

  return SUCCESS;
}

bool IsGeOp(const ge::OpDescPtr &op_desc_ptr) {
  bool is_ge_op = false;
  if (ge::AttrUtils::GetBool(op_desc_ptr, IS_GE_OP, is_ge_op) && is_ge_op) {
    FE_LOGI("Op %s is ge op!", op_desc_ptr->GetName().c_str());
    return true;
  }
  return false;
}

void FEOpsKernelInfoStore::SetCheckSupportedStaticFlag(bool stat_flag) { check_support_static_flag_ = stat_flag; }

bool FEOpsKernelInfoStore::CheckSupported(const ge::OpDescPtr &op_desc_ptr, std::string &un_supported_reason) const {
  FE_LOGD("Node %s begin CheckSupported.", op_desc_ptr->GetName().c_str());
  /* Op Cast need to be checked accurately, because the input and output should
   * be matched correspondingly */
  int64_t start_usec_CheckSupported = 0;
  if (check_support_static_flag_) {
    start_usec_CheckSupported = GetMicroSecondTime();
  }

  bool ret = false;
  if (op_desc_ptr->GetType() == CAST) {
    ret = CheckSupportedBase(op_desc_ptr, un_supported_reason, ACCURACY_MODE, true);
  } else {
    ret = CheckSupportedBase(op_desc_ptr, un_supported_reason, DTYPE_FORMAT_MODE, true);
  }
  if (check_support_static_flag_) {
    FE_TIMECOST_END(CheckSupported, "FEOpsKernelInfoStore::CheckSupported");
  }

  return ret;
}

uint64_t GenerateHashKey(const ge::OpDescPtr &op_desc_ptr) {
  int64_t src_format = 0;
  int64_t dst_format = 0;
  (void)ge::AttrUtils::GetInt(op_desc_ptr, ATTR_NAME_DST_FORMAT, dst_format);
  (void)ge::AttrUtils::GetInt(op_desc_ptr, ATTR_NAME_SRC_FORMAT, src_format);

  /* hash key is 64-bits unsigned integer which consists of 8 bits of input0's
   * format (bit 0 to 7), 8 bits of input0's data type (bit 8 to 15), 8
   * bits of output0's format (bit 16 to 23), 8 bits of output0's data type
   * (bit 24 to 31), 8 bits of attribute src_format (bit 32 to 39) and 8 bits
   * attribute dst_format (bbit 40 to 47). The last 16 bits is reserved. */

  uint64_t input0_format = (uint64_t)ge::GetPrimaryFormat(op_desc_ptr->GetInputDescPtr(0)->GetFormat());
  uint64_t input0_data_type = (uint64_t)op_desc_ptr->GetInputDescPtr(0)->GetDataType();
  uint64_t output0_format = (uint64_t)ge::GetPrimaryFormat(op_desc_ptr->GetOutputDescPtr(0)->GetFormat());
  uint64_t output0_data_type = (uint64_t)op_desc_ptr->GetOutputDescPtr(0)->GetDataType();

  uint64_t hash_key = ((input0_format) | (input0_data_type << BIT_SHIFT_8) | (output0_format << BIT_SHIFT_16) |
                       (output0_data_type << BIT_SHIFT_24) | ((uint64_t)src_format << BIT_SHIFT_32) |
                       ((uint64_t)dst_format << BIT_SHIFT_40));
  return hash_key;
}

bool FEOpsKernelInfoStore::CheckAccuracySupportByCache(const ge::OpDescPtr &op_desc_ptr) {
  uint64_t hash_key = GenerateHashKey(op_desc_ptr);
  auto iter = checksupport_cache_.find(hash_key);
  if (iter != checksupport_cache_.end()) {
    FE_LOGD("op %s's hash key %lx is found in cache.", op_desc_ptr->GetName().c_str(), hash_key);
    if (iter->second.result == true) {
      ge::AttrUtils::SetInt(op_desc_ptr, FE_IMPLY_TYPE, iter->second.fe_impl_type);
      ge::AttrUtils::SetInt(op_desc_ptr, ge::ATTR_NAME_IMPLY_TYPE, iter->second.ge_impl_type);
      return true;
    }
  }
  return false;
}

Status FEOpsKernelInfoStore::StoreCheckSuportResultForTransNodes(const ge::OpDescPtr &op_desc_ptr, bool result) {
  uint64_t hash_key = GenerateHashKey(op_desc_ptr);
  int64_t fe_impl_type = 0;
  int64_t ge_impl_type = 0;
  (void)ge::AttrUtils::GetInt(op_desc_ptr, FE_IMPLY_TYPE, fe_impl_type);
  (void)ge::AttrUtils::GetInt(op_desc_ptr, ge::ATTR_NAME_IMPLY_TYPE, ge_impl_type);

  CheckSuportResult check_suport_result = {result, fe_impl_type, ge_impl_type};
  checksupport_cache_.emplace(std::make_pair(hash_key, check_suport_result));
  FE_LOGD("Store op %s's {hash key %lx, result %u, impltype %u, %u}into cache.", op_desc_ptr->GetName().c_str(),
          hash_key, result, fe_impl_type, ge_impl_type);
  return SUCCESS;
}

bool FEOpsKernelInfoStore::CheckAccuracySupported(const ge::OpDescPtr &op_desc_ptr, std::string &un_supported_reason,
                                                  bool real_query) const {
  FE_CHECK(op_desc_ptr == nullptr,
           REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] op_desc_ptr is null."), return false);
  FE_LOGD("Node %s begin CheckAccuracySupported.", op_desc_ptr->GetName().c_str());
  bool ret = CheckSupportedBase(op_desc_ptr, un_supported_reason, ACCURACY_MODE, real_query);

  return ret;
}

bool FEOpsKernelInfoStore::CheckSupportedBase(const ge::OpDescPtr &op_desc_ptr, std::string &un_supported_reason,
                                              CheckSupportMode mode, bool real_query) const {
  FE_CHECK(op_store_adapter_manager_ptr_ == nullptr, REPORT_FE_ERROR("[GraphOpt][Setcheck] \
           op_store_adapter_manager_ptr_ is null."),
           return false);

  int64_t is_check_supported = 0;
  std::ostringstream reason_oss;
  std::ostringstream ge_reason_oss;
  /* real_query = true means we need skip the cache value of IS_CHECK_SUPPORTED
   * and do the checksupport again */
  if (!real_query && ge::AttrUtils::GetInt(op_desc_ptr, IS_CHECK_SUPPORTED, is_check_supported)) {
    /* The highest bit of "isCheckSupported" indicates whether or not it is
     * supported, 0: supported 1: not supported.
     **/
    uint64_t reason = static_cast<uint64_t>(is_check_supported);
    if ((reason & NOT_SUPPORTED_FLAG_BIT) == 0) {
      FE_LOGD("Node %s has been check_supported, result is support.", op_desc_ptr->GetName().c_str());
      return true;
    } else {
      FE_LOGI_IF(GetNotSupportedReasonByAttr(reason, ge_reason_oss) != SUCCESS,
                 "Get Node %s not supported reason not success.", op_desc_ptr->GetName().c_str());
      un_supported_reason = ge_reason_oss.str();
      FE_LOGI("Node %s has been check_supported, result is not support.", op_desc_ptr->GetName().c_str());
      return false;
    }
  }

  OpImplType imply_type = EN_RESERVED;
  UnSupportedReason reason;
  if (CheckSupportedByOpsStore(op_desc_ptr, reason, imply_type, mode)) {
    FE_LOGD("[ChkSpt][FEChk][Node %s, type %s] This op is supported by FE with impl type %u in mode %u.",
            op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str(), imply_type, mode);
    // 1. check the imply_type
    auto iter = IMPL_TYPE_MAP.find(imply_type);
    if (iter == IMPL_TYPE_MAP.end()) {
      REPORT_FE_ERROR(
          "[GraphOpt][Setcheck] Op[name=%s,type=%s]: the FE imply type and GE imply type map is not found, \
          the FE impl_type is [%d].",
          op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str(), imply_type);
      return false;
    }

    // 2. set the fe and ge imply type of the op
    ge::AttrUtils::SetInt(op_desc_ptr, FE_IMPLY_TYPE, static_cast<int>(imply_type));
    FE_LOGD("Op[name=%s,type=%s]: set the FE_IMPLY_TYPE attribute [%s], set the IMPLY_TYPE attribute [%s].",
            op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str(), GetImplTypeString(imply_type).c_str(),
            GetGeImplTypeString(iter->second).c_str());
    /* If the op is ge op, we only use the ops kernel info to select its
     * format and data type. Actually there is no op implementation for this op.
     * So we return false and then GE will not distribute this op to FE to
     * precompile and compile. */
    bool l1_reshape_condition =
        (op_desc_ptr->GetType() == RESHAPE && Configuration::Instance(engine_name_).EnableL1Fusion());
    if (IsGeOp(op_desc_ptr) && !l1_reshape_condition) {
      FE_LOGI("FE does not support Ge Op [%s,%s], although it's in ops info library.", op_desc_ptr->GetName().c_str(),
              op_desc_ptr->GetType().c_str());
      return false;
    }

    ge::AttrUtils::SetInt(op_desc_ptr, ge::ATTR_NAME_IMPLY_TYPE, static_cast<int>(iter->second));
    return true;
  }

  FE_LOGI_IF(GetNotSupportedReasonByAttr(reason.reason_id, ge_reason_oss) != SUCCESS,
             "Get Node %s not supported reason not success.", op_desc_ptr->GetName().c_str());
  un_supported_reason = reason.reason;
  FE_LOGW("This Op %s is not supported in all op information librarys.", op_desc_ptr->GetType().c_str());

  return false;
}

Status FEOpsKernelInfoStore::GetHighPrioOpKernelInfoPtr(const string &op_type, OpKernelInfoPtr &op_kernel_info_ptr) {
  std::vector<FEOpsStoreInfo> fe_ops_store_info_vec = Configuration::Instance(engine_name_).GetOpsStoreInfo();
  for (auto &ops_store : fe_ops_store_info_vec) {
    OpKernelInfoPtr op_kernel_ptr =
        OpsKernelManager::Instance(engine_name_).GetOpKernelInfoByOpType(ops_store.fe_ops_store_name, op_type);
    if (op_kernel_ptr == nullptr) {
      continue;
    }
    op_kernel_info_ptr = op_kernel_ptr;
    op_kernel_info_ptr->GetOpInfo().opKernelLib = ops_store.fe_ops_store_name;
    return SUCCESS;
  }
  FE_LOGW("Can not find Op (type [%s]) in all ops information libs.", op_type.c_str());
  return OP_NOT_FOUND_IN_QUERY_HIGH_PRIO_IMPL;
}

Status FEOpsKernelInfoStore::GetAllSubOpsStore(std::map<std::string, SubOpsStorePtr> &all_sub_store_ptr) const {
  all_sub_store_ptr = map_all_sub_store_info_;
  return SUCCESS;
}

Status FEOpsKernelInfoStore::QueryHighPrioOpImplType(const ge::OpDescPtr &op_desc_ptr, OpImplType &impl_type) {
  FE_LOGD("QueryHighPrioOpImplType, querying the highest implement type of Op %s.", op_desc_ptr->GetType().c_str());
  UnSupportedReason reason;
  /* Op Cast need to be checked accurately, because the input and output should
   * be matched correspondingly */
  if (op_desc_ptr->GetType() == CAST) {
    if (!CheckSupportedByOpsStore(op_desc_ptr, reason, impl_type, ACCURACY_MODE)) {
      FE_LOGW("Op %s, type %s is not supported in all op information librarys in accurate mode.",
              op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str());
      return OP_NOT_FOUND_IN_QUERY_HIGH_PRIO_IMPL;
    }
  } else {
    if (!CheckSupportedByOpsStore(op_desc_ptr, reason, impl_type, DTYPE_FORMAT_MODE)) {
      FE_LOGW("This Op %s is not supported in all op information librarys.", op_desc_ptr->GetType().c_str());
      return OP_NOT_FOUND_IN_QUERY_HIGH_PRIO_IMPL;
    }
  }

  return SUCCESS;
}

Status FEOpsKernelInfoStore::CompileOpGetImplType(ge::OpDescPtr op_desc_ptr, OpImplType &impl_type) {
  impl_type = EN_RESERVED;
  int tmp_imply_type = -1;
  if (!ge::AttrUtils::GetInt(op_desc_ptr, FE_IMPLY_TYPE, tmp_imply_type)) {
    // chose high prior op info lib for node
    if (QueryHighPrioOpImplType(op_desc_ptr, impl_type) == SUCCESS) {
      ge::AttrUtils::SetInt(op_desc_ptr, FE_IMPLY_TYPE, static_cast<int>(impl_type));
      FE_LOGD("Set FE op imply type [%d] of OP [%s].", impl_type, op_desc_ptr->GetName().c_str());
    } else {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompOpGetImplType] OP [%s] is not supported by FE.",
                      op_desc_ptr->GetName().c_str());
      return FAILED;
    }

    // set ge imply type
    ge::AttrUtils::SetInt(op_desc_ptr, ge::ATTR_NAME_IMPLY_TYPE, static_cast<int>(impl_type));
    FE_LOGD("Set node name %s GE imply type, %s = %d.", op_desc_ptr->GetName().c_str(),
            ge::ATTR_NAME_IMPLY_TYPE.c_str(), impl_type);
  } else {
    impl_type = (OpImplType)tmp_imply_type;
  }

  return SUCCESS;
}

Status FEOpsKernelInfoStore::CompileOpGetTvmJsonInfo(ScopeNodeIdMap &fusion_nodes_map, ScopeJsonMap_t &scope_json_map) {
  // PreCompile tbe-builtin op
  for (auto &fusion_item : fusion_nodes_map) {
    int64_t scope_id = fusion_item.first;
    ge::Node &node = *((fusion_item.second)[0]);
    ge::OpDescPtr op_desc_ptr = node.GetOpDesc();
    // find the json file path according to scope id
    auto json_iter = scope_json_map.find(scope_id);
    if (json_iter == scope_json_map.end()) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompOpGetTvmJsInfo] scopeId = %ld, op name = %s, op type = %s, \
                      json file is not found.",
                      scope_id, op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str());
      std::map<std::string, std::string> error_key_map;
      error_key_map[EM_OP_NAME] = op_desc_ptr->GetName();
      error_key_map[EM_ERROR_MSG] = "Compile json file is not found.";
      LogErrorMessage(EM_GET_FILEPATH_FAILED, error_key_map);
      return OP_COMPILER_CHECK_FALSE_FAILED;
    }
    op_desc_ptr->DelAttr(SCOPE_ID_ATTR);
    op_desc_ptr->DelAttr(L1_SCOPE_ID_ATTR);

    std::string json_file_path = json_iter->second;
    TbeJsonInfo info;
    info.json_file_path = json_file_path;
    // package tvm json info
    TbeJsonFileParsePtr parse_ptr = nullptr;
    FE_MAKE_SHARED(parse_ptr = std::make_shared<TbeJsonFileParse>(op_desc_ptr),
                   return fe::OP_COMPILER_MAKE_SHARED_FAILED);
    if (parse_ptr->PackageTvmJsonInfo(info) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompOpGetTvmJsInfo] PackageTvmJsonInfo failed, op name = %s, \
                      op type = %s.", op_desc_ptr->GetName().c_str(),
                      op_desc_ptr->GetType().c_str());
      return FAILED;
    }
  }
  return SUCCESS;
}

std::vector<uint32_t> FEOpsKernelInfoStore::CompileGetAtomicOutput(ge::OpDescPtr &op_desc_ptr) {
  std::vector<uint32_t> output_index;
  std::vector<uint32_t> tmp_output_index;
  if (ge::AttrUtils::GetListInt(op_desc_ptr, TBE_OP_ATOMIC_OUTPUT_INDEX, tmp_output_index)) {
    uint32_t output_size = tmp_output_index.size();
    for (uint32_t i = 0; i < output_size; i++) {
      // get atomic output index
      if (tmp_output_index[i] == 1) {
        output_index.push_back(i);
      }
    }
  }

  return output_index;
}

Status FEOpsKernelInfoStore::CompileSetAtomicCleanWorkSpace(ge::OpDescPtr &op_desc_ptr, vector<int64_t> &work_space,
                                                            vector<int64_t> &work_space_bytes) {
  std::vector<uint32_t> output_index;
  output_index = CompileGetAtomicOutput(op_desc_ptr);
  for (auto &idx : output_index) {
    work_space.push_back(0);
    if (TensorSizeCalculator::CalculateOpTensorSize(*op_desc_ptr) != SUCCESS) {
      return FAILED;
    }
    int64_t tensor_size = 0;
    (void)ge::TensorUtils::GetSize(op_desc_ptr->GetOutputDesc(idx), tensor_size);

    work_space_bytes.push_back(tensor_size);
    FE_LOGD("Get op:%s, idx:%u, tensor_size:%ld", op_desc_ptr->GetName().c_str(), idx, tensor_size);
  }

  uint32_t atomic_flag = 0;
  if (ge::AttrUtils::GetInt(op_desc_ptr, TBE_OP_ATOMIC_WORKSPACE_FLAG, atomic_flag)) {
    if (atomic_flag == 1) {
      std::vector<int64_t> workspace_bytes_vec = op_desc_ptr->GetWorkspaceBytes();
      int64_t total_wk_space_bytes = accumulate(workspace_bytes_vec.begin(), workspace_bytes_vec.end(), 0);
      work_space.push_back(0);
      work_space_bytes.push_back(total_wk_space_bytes);
      FE_LOGD("Get op:%s, ta:%ld", op_desc_ptr->GetName().c_str(), total_wk_space_bytes);
    }
  }

  return SUCCESS;
}

Status FEOpsKernelInfoStore::GetAllAtomicCleanNode(ge::NodePtr &node_ptr, vector<ge::NodePtr> &atomic_node_vec) {
  FE_CHECK_NOTNULL(node_ptr);
  /* if the node is atomic node and connected to netoutput,
     then add it into atomic_node_vec and continue to next node */
  ge::OpDescPtr op_desc_ptr = node_ptr->GetOpDesc();
  FE_CHECK_NOTNULL(op_desc_ptr);
  bool is_atomic_node = false;
  bool is_net_output = false;
  if (ge::AttrUtils::GetBool(op_desc_ptr, ge::ATOMIC_ATTR_IS_ATOMIC_NODE, is_atomic_node) &&
      ge::AttrUtils::GetBool(op_desc_ptr, ATTR_NAME_IS_CONNECTED_TO_NETOUTPUT, is_net_output)) {
    if (is_atomic_node && is_net_output) {
      atomic_node_vec.push_back(node_ptr);
      FE_LOGD(
          "op:%s is atomic node and connected to the netoutput, do not need"
          "to be compiled, will just create and compile an atomic clean node for it.",
          node_ptr->GetName().c_str());
    }
  }

  // add atomic clean node, and compile.
  FE_LOGD("Op[name:%s,type:%s] is belong to unknown graph.", op_desc_ptr->GetName().c_str(),
          op_desc_ptr->GetType().c_str());

  bool atomic_node_flag = false;
  if (SetAtomicOpAttr(op_desc_ptr, atomic_node_flag) == FAILED) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][GetAllAtoCleanNd] op:%s, set atomic info fail",
                    node_ptr->GetName().c_str());
    return FAILED;
  }
  if (atomic_node_flag) {
    atomic_node_vec.push_back(node_ptr);
  }
  return SUCCESS;
}

Status SetWorkSpaceForAtomicClean(const vector<int64_t> &work_space, const vector<int64_t> &work_space_bytes,
                                  const ge::OpDescPtr &op_desc_ptr, const ge::ComputeGraphPtr &tmp_graph,
                                  vector<ge::NodePtr> &atomic_clean_nodes) {
  if (!work_space.empty()) {
    ge::OpDescPtr atomic_clean_op_desc_ptr = nullptr;
    bool is_unknown_shape_op = IsFeSupportedDynamicOp(*op_desc_ptr);
    FE_LOGD("Op[name:%s,type:%s] unknown shape flag is %d.", op_desc_ptr->GetName().c_str(),
            op_desc_ptr->GetType().c_str(), is_unknown_shape_op);
    if (is_unknown_shape_op) {
      FE_MAKE_SHARED(
          atomic_clean_op_desc_ptr = make_shared<ge::OpDesc>("atomic_clean_node", DYNAMIC_ATOMIC_CLEAN_OP_TYPE),
          return FAILED);
      (void)ge::AttrUtils::SetBool(atomic_clean_op_desc_ptr, STR_SUPPORT_DYNAMIC_SHAPE, true);
      (void)ge::AttrUtils::SetInt(atomic_clean_op_desc_ptr, ATTR_NAME_IS_UNKNOWN_SHAPE_OP, IS_UNKNOWN_SHAPE_VALUE);
      (void)ge::AttrUtils::SetInt(atomic_clean_op_desc_ptr, ATTR_NAME_IS_UNKNOWN_GRAPH, IS_UNKNOWN_SHAPE_VALUE);
    } else {
      FE_MAKE_SHARED(atomic_clean_op_desc_ptr = make_shared<ge::OpDesc>("atomic_clean_node", ATOMIC_CLEAN_OP_TYPE),
                     return FAILED);
    }

    string name = atomic_clean_op_desc_ptr->GetName() + to_string(GetAtomicId());
    atomic_clean_op_desc_ptr->SetName(name);
    atomic_clean_op_desc_ptr->SetWorkspace(work_space);
    atomic_clean_op_desc_ptr->SetWorkspaceBytes(work_space_bytes);
    ge::AttrUtils::SetListInt(atomic_clean_op_desc_ptr, ge::ATTR_NAME_AUTOMIC_ADD_START, work_space);
    ge::AttrUtils::SetListInt(atomic_clean_op_desc_ptr, ge::ATTR_NAME_AUTOMIC_ADD_MEM_SIZE, work_space_bytes);
    ge::AttrUtils::SetInt(atomic_clean_op_desc_ptr, FE_IMPLY_TYPE, EN_IMPL_HW_TBE);

    ge::NodePtr atomic_clean_node = tmp_graph->AddNode(atomic_clean_op_desc_ptr, op_desc_ptr->GetId());
    FE_CHECK_NOTNULL(atomic_clean_node);
    atomic_clean_nodes.push_back(atomic_clean_node);
    op_desc_ptr->SetExtAttr(ATTR_NAME_ATOMIC_CLEAN_NODE, atomic_clean_node);
    FE_LOGD("Create atomic clean op:%s for op:%s, work_space_vec_num:%zu", name.c_str(), op_desc_ptr->GetName().c_str(),
            work_space.size());
  }
  return SUCCESS;
}

Status FEOpsKernelInfoStore::CompileAndSetKernelNameForAtomicClean(const vector<ge::NodePtr> &node_vec,
                                                                   vector<ge::NodePtr> &atomic_clean_nodes) {
  if (!atomic_clean_nodes.empty()) {
    if (FEOpsKernelInfoStore::CompileOp(atomic_clean_nodes) != SUCCESS) {
      return FAILED;
    }
    /* link the atomic clean node bin file to the corresponding atomic node with
       the extra attribute tbe_atomic_kernel */
    for (auto &node_ptr : node_vec) {
      ge::OpDescPtr op_desc_ptr = node_ptr->GetOpDesc();
      ge::NodePtr atomic_clean_node = nullptr;
      atomic_clean_node = op_desc_ptr->TryGetExtAttr(ATTR_NAME_ATOMIC_CLEAN_NODE, atomic_clean_node);
      FE_CHECK(atomic_clean_node == nullptr,
               REPORT_FE_ERROR("[SubGraphOpt][Compile][CompSetAtomCleanNm] get node:%s's extra attribute \
               atomic_clean_node_ptr failed.", op_desc_ptr->GetName().c_str()), return FAILED);
      ge::OpKernelBinPtr tbe_kernel_ptr = nullptr;
      tbe_kernel_ptr = atomic_clean_node->GetOpDesc()->TryGetExtAttr(ge::OP_EXTATTR_NAME_TBE_KERNEL, tbe_kernel_ptr);
      FE_CHECK(tbe_kernel_ptr == nullptr,
               REPORT_FE_ERROR("[SubGraphOpt][Compile][CompSetAtomCleanNm] Get node:%s's extra attribute tbe_kernel \
               failed.", atomic_clean_node->GetName().c_str()), return FAILED);
      op_desc_ptr->SetExtAttr(TBE_ATOMIC_KERNEL, tbe_kernel_ptr);
      FE_LOGD("Set node:%s's extra attribute tbe_atomic_kernel success.", op_desc_ptr->GetName().c_str());
      int64_t atomic_op_para_size = 0;
      ge::AttrUtils::GetInt(atomic_clean_node->GetOpDesc(), OP_PARA_SIZE, atomic_op_para_size);
      ge::AttrUtils::SetInt(op_desc_ptr, ATOMIC_OP_PARA_SIZE, (int64_t)atomic_op_para_size);

      // set compile info
      std::string compile_info_json;
      if (ge::AttrUtils::GetStr(atomic_clean_node->GetOpDesc(), COMPILE_INFO_JSON, compile_info_json)) {
        (void)ge::AttrUtils::SetStr(op_desc_ptr, ATOMIC_COMPILE_INFO_JSON, compile_info_json);
      }
      std::string compile_info_key;
      if (ge::AttrUtils::GetStr(atomic_clean_node->GetOpDesc(), COMPILE_INFO_KEY, compile_info_key)) {
        (void)ge::AttrUtils::SetStr(op_desc_ptr, ATOMIC_COMPILE_INFO_KEY, compile_info_key);
      }

      // Set attr atomic_kernelname
      string AttrKeyKernelName = op_desc_ptr->GetName() + "_kernelname";
      string AttrKeyAtomicKernelName = op_desc_ptr->GetName() + "_atomic_kernelname";
      string AttrValKernelName;
      if (ge::AttrUtils::GetStr(atomic_clean_node->GetOpDesc(), AttrKeyKernelName, AttrValKernelName)) {
        ge::AttrUtils::SetStr(op_desc_ptr, AttrKeyAtomicKernelName, AttrValKernelName);
        FE_LOGD("set node[name:%s,type:%s]'s attribute atomic_kernelname success.", op_desc_ptr->GetName().c_str(),
                op_desc_ptr->GetType().c_str());
      } else {
        FE_LOGD("Get node[name:%s,type:%s]'s attribute kernelname failed.", atomic_clean_node->GetName().c_str(),
                atomic_clean_node->GetType().c_str());
      }
    }
  }
  return SUCCESS;
}

Status FEOpsKernelInfoStore::CompileAtomicClean(vector<ge::NodePtr> &node_vec) {
  vector<ge::NodePtr> atomic_clean_nodes;
  ge::ComputeGraphPtr tmp_graph;
  FE_MAKE_SHARED((tmp_graph = std::make_shared<ge::ComputeGraph>("OpCompileGraph")), return FAILED);

  for (auto &node_ptr : node_vec) {
    ge::OpDescPtr op_desc_ptr = node_ptr->GetOpDesc();
    vector<int64_t> work_space;
    vector<int64_t> work_space_bytes;

    if (CompileSetAtomicCleanWorkSpace(op_desc_ptr, work_space, work_space_bytes) == FAILED) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompAtomClean] op:%s set work_space failed",
                      op_desc_ptr->GetName().c_str());
    }

    // set atomic clean op work_space and work_space_bytes
    if (SetWorkSpaceForAtomicClean(work_space, work_space_bytes, op_desc_ptr, tmp_graph, atomic_clean_nodes) !=
        SUCCESS) {
      return FAILED;
    }
  }

  // atomic clean to compile
  if (CompileAndSetKernelNameForAtomicClean(node_vec, atomic_clean_nodes) != SUCCESS) {
    return FAILED;
  }
  return SUCCESS;
}

Status FEOpsKernelInfoStore::PrePareCompileParameter(
    ge::Node &node, string &op_type, OpImplType &impl_type,
    std::unordered_map<OpStoreAdapterPtr, vector<PreCompileNodePara>> &node_map) {
  ge::OpDescPtr op_desc_ptr = node.GetOpDesc();

  // caculate input output size
  FE_CHECK_NOTNULL(op_store_adapter_manager_ptr_);
  OpStoreAdapterPtr op_store_adapter;
  if (SUCCESS != op_store_adapter_manager_ptr_->GetOpStoreAdapter(impl_type, op_store_adapter)) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][PrepCompParm] Failed to get op information library adapter by \
                    op_imply_type[%s].", GetImplTypeString(impl_type).c_str());
    return FAILED;
  }

  FEOpsStoreInfo op_store_info;
  if (Configuration::Instance(engine_name_).GetOpStoreInfoByImplType(impl_type, op_store_info) != SUCCESS) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][PrepCompParm] Failed to get op information library info by impl_type[%s].",
                    GetImplTypeString(impl_type).c_str());
    return OP_COMPILER_CHECK_FALSE_FAILED;
  }
  std::string imply_type_str = op_store_info.fe_ops_store_name;

  // get registered precompile function
  OpKernelInfoPtr op_kernel_info_ptr =
      OpsKernelManager::Instance(engine_name_).GetOpKernelInfoByOpType(imply_type_str, op_type);
  if (!op_desc_ptr->HasAttr(ge::ATTR_NAME_UNREGST_OPPATH)) {
    FE_CHECK(op_kernel_info_ptr == nullptr,
             REPORT_FE_ERROR("[SubGraphOpt][Compile][PrepCompParm] opKernelInfoPtr is nullptr."), return FAILED);
  }

  bool is_custom_op = false;
  (void)ge::AttrUtils::GetBool(op_desc_ptr, NON_PERSISTENT_CUSTOM_OP_FLAG, is_custom_op);
  std::string op_dsl_file_path;
  bool ret_status = (is_custom_op && op_kernel_info_ptr != nullptr && !op_kernel_info_ptr->GetOpImpPath().empty());
  if (op_desc_ptr->HasAttr(ge::ATTR_NAME_UNREGST_OPPATH)) {
    if (!ge::AttrUtils::GetStr(op_desc_ptr, ge::ATTR_NAME_UNREGST_OPPATH, op_dsl_file_path)) {
      FE_LOGI("Fail to get attr:_unregst_oppath of node[%s]", op_desc_ptr->GetName().c_str());
      return FAILED;
    }
  } else if (ret_status) {
    op_dsl_file_path = op_kernel_info_ptr->GetOpImpPath();
  } else {
    op_dsl_file_path = op_store_info.op_impl_file_path;
  }

  bool is_tbe = impl_type == EN_IMPL_CUSTOM_TBE || impl_type == EN_IMPL_PLUGIN_TBE || impl_type == EN_IMPL_HW_TBE ||
                impl_type == EN_IMPL_NON_PERSISTENT_CUSTOM_TBE || impl_type == EN_IMPL_VECTOR_CORE_HW_TBE ||
                impl_type == EN_IMPL_VECTOR_CORE_CUSTOM_TBE;

  if (is_tbe) {
    string session_graph_id;
    if (!ge::AttrUtils::GetStr(node.GetOpDesc(), ge::ATTR_NAME_SESSION_GRAPH_ID, session_graph_id)) {
      FE_LOGW("%s get session_graph_id failed", node.GetName().c_str());
    }

    PreCompileNodePara pre_comp_node_para = {&node, op_kernel_info_ptr, imply_type_str, op_dsl_file_path,
                                             session_graph_id};
    if (node_map.find(op_store_adapter) == node_map.end()) {
      vector<PreCompileNodePara> pre_comp_node_para_vec;
      pre_comp_node_para_vec.push_back(pre_comp_node_para);
      node_map.emplace(make_pair(op_store_adapter, pre_comp_node_para_vec));
    } else {
      node_map[op_store_adapter].push_back(pre_comp_node_para);
    }
  }
  return SUCCESS;
}

Status FEOpsKernelInfoStore::PreCompileAndCompile(
    std::unordered_map<OpStoreAdapterPtr, vector<PreCompileNodePara>> &node_map, const ge::NodePtr &node,
    ScopeNodeIdMap &fusion_node_map) {
  for (auto &comp_para : node_map) {
    OpStoreAdapterPtr op_store_adapter = comp_para.first;
    if (op_store_adapter->PreCompileOp(comp_para.second) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][PrepCompAndComp] PreCompileOp failed");
      return FAILED;
    }
  }
  // get tbe adapter
  OpStoreAdapterPtr op_store_adapter = nullptr;
  if (SUCCESS != op_store_adapter_manager_ptr_->GetOpStoreAdapter(EN_IMPL_HW_TBE, op_store_adapter)) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][PrepCompAndComp] Failed to get op info library adapter by \
                    op_imply_type[%d].", EN_IMPL_HW_TBE);
    return FAILED;
  }

  // Compile tbe op
  ScopeJsonMap_t scope_json_map;
  std::vector<ge::NodePtr> buff_fus_compile_failed_nodes;
  std::vector<ge::NodePtr> buff_fus_to_del_nodes;
  if (op_store_adapter->CompileOp(fusion_node_map, scope_json_map, buff_fus_compile_failed_nodes,
                                  buff_fus_to_del_nodes) != SUCCESS) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][PrepCompAndComp] CompileOp failed, node name = %s.",
                    !(node == nullptr) ? node->GetName().c_str() : "");
    return FAILED;
  }

  if (CompileOpGetTvmJsonInfo(fusion_node_map, scope_json_map) == FAILED) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][PrepCompAndComp] CompileOp failed, node name = %s.",
                    !(node == nullptr) ? node->GetName().c_str() : "");
    return FAILED;
  }

  return SUCCESS;
}

Status FEOpsKernelInfoStore::CompileSingleOp(ge::NodePtr &node_ptr) {
  ScopeNodeIdMap fusion_node_map;
  int64_t scope_id = -1;
  std::unordered_map<OpStoreAdapterPtr, vector<PreCompileNodePara>> node_map;
  vector<ge::NodePtr> atomic_node_vec;
  auto &node = *node_ptr;
  string op_type = node.GetType();
  ge::OpDescPtr op_desc_ptr = node.GetOpDesc();
  FE_CHECK_NOTNULL(op_desc_ptr);

  string const_op_type;
  bool const_flag = ge::NodeUtils::GetConstOpType(node_ptr, const_op_type);
  if ((op_type == OP_TYPE_PLACE_HOLDER || op_type == OP_TYPE_END || op_type == "Data" || const_flag)) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][CompSingleOp] Compile single op failed, op %s not supported.",
                    op_type.c_str());
    return FAILED;
  }

  // if the node is atomic node and connected to netoutput, then add it into atomic_node_vec and continue to next node
  bool is_atomic_node = false;
  bool is_net_output = false;
  if (ge::AttrUtils::GetBool(op_desc_ptr, ge::ATOMIC_ATTR_IS_ATOMIC_NODE, is_atomic_node) &&
      ge::AttrUtils::GetBool(op_desc_ptr, "is_connected_to_netoutput", is_net_output)) {
    if (is_atomic_node && is_net_output) {
      if (op_type == "AtomicAddrClean") {
        REPORT_FE_ERROR("[SubGraphOpt][Compile][CompSingleOp] op:%s is atomic clean node, but \
                        is_connected_to_netoutput are both true.", node.GetName().c_str());
        return FAILED;
      }
      if (CompileAtomicClean(atomic_node_vec) != SUCCESS) {
        return FAILED;
      }
      FE_LOGI("op:%s is atomic and connected to the netoutput, only compile atomic clean.", node.GetName().c_str());
    }
  }

  if (!IsNeededCompile(op_desc_ptr)) {
    return SUCCESS;
  }

  OpImplType op_impl_type = EN_IMPL_HW_TBE;
  if (PrePareCompileParameter(node, op_type, op_impl_type, node_map) != SUCCESS) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][CompSingleOp] PreCompile single op failed, op %s not supported.",
                    op_type.c_str());
    return FAILED;
  }

  std::vector<ge::Node *> node_list_new;
  node_list_new.push_back(&node);
  fusion_node_map.emplace(std::pair<int64_t, std::vector<ge::Node *>>(scope_id, node_list_new));
  scope_id--;

  return PreCompileAndCompile(node_map, node_ptr, fusion_node_map);
}

void FillFusionNodeMapForTbeOp(ge::Node &node, OpImplType impl_type, ScopeNodeIdMap &fusion_node_map,
                               int64_t &scope_id) {
  bool impl_type_check = (impl_type == EN_IMPL_CUSTOM_TBE || impl_type == EN_IMPL_HW_TBE ||
                          impl_type == EN_IMPL_NON_PERSISTENT_CUSTOM_TBE || impl_type == EN_IMPL_VECTOR_CORE_HW_TBE ||
                          impl_type == EN_IMPL_VECTOR_CORE_CUSTOM_TBE);
  if (impl_type_check) {
    std::vector<ge::Node *> node_list_new;
    node_list_new.push_back(&node);
    fusion_node_map.emplace(std::pair<int64_t, std::vector<ge::Node *>>(scope_id, node_list_new));
    scope_id--;
  }
}

Status FEOpsKernelInfoStore::CompileMultipleOp(vector<ge::NodePtr> &node_vec) {
  ScopeNodeIdMap fusion_node_map;
  int64_t scope_id = -1;
  std::unordered_map<OpStoreAdapterPtr, vector<PreCompileNodePara>> node_map;
  vector<ge::NodePtr> atomic_node_vec;
  if (node_vec.empty()) {
    FE_LOGW("node vector is empty.");
    return SUCCESS;
  }

  for (auto &node_ptr : node_vec) {
    auto &node = *node_ptr;
    string op_type = node.GetType();
    ge::OpDescPtr op_desc_ptr = node.GetOpDesc();
    FE_CHECK_NOTNULL(op_desc_ptr);

    if (!IsNeededCompile(op_desc_ptr)) {
      continue;
    }

    string const_op_type;
    bool const_flag = ge::NodeUtils::GetConstOpType(node_ptr, const_op_type);
    if ((op_type == OP_TYPE_PLACE_HOLDER || op_type == OP_TYPE_END || op_type == "Data" || const_flag)) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompMultiOp] Compile single op failed, op %s not supported.",
                      op_type.c_str());
      return FAILED;
    }

    OpImplType impl_type = EN_RESERVED;
    if (CompileOpGetImplType(op_desc_ptr, impl_type) == FAILED) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompMultiOp] Compile single op %s failed, get impl_type failed.",
                      op_type.c_str());
      return FAILED;
    }

    if (PrePareCompileParameter(node, op_type, impl_type, node_map) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompMultiOp] PreCompile single op failed, op %s not supported.",
                      op_type.c_str());
      return FAILED;
    }

    FillFusionNodeMapForTbeOp(node, impl_type, fusion_node_map, scope_id);
  }

  if (PreCompileAndCompile(node_map, node_vec[0], fusion_node_map) != SUCCESS) {
    return FAILED;
  }

  for (auto &node_ptr : node_vec) {
    if (GetAllAtomicCleanNode(node_ptr, atomic_node_vec) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompMultiOp] Get atomic node for op[name:%s,type:%s] failed.",
                      node_ptr->GetOpDesc()->GetName().c_str(), node_ptr->GetOpDesc()->GetType().c_str());
      return FAILED;
    }
  }
  // create and compile atomic clean node for node in atomic_node_vec
  if (!atomic_node_vec.empty() && CompileAtomicClean(atomic_node_vec) != SUCCESS) {
    return FAILED;
  }
  return SUCCESS;
}

Status FEOpsKernelInfoStore::CompileOp(vector<ge::NodePtr> &node_vec) {
  FE_TIMECOST_START(FEOpsKernelCompileOp);
  if (node_vec.empty()) {
    FE_LOGD("No nodes need to do compile.");
    return SUCCESS;
  }
  if (node_vec[0]->GetOpDesc()->HasAttr(ge::ATTR_NAME_UNREGST_OPPATH)) {
    if (CompileSingleOp(node_vec[0]) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompileOp] Fail to compile node[%s].", node_vec[0]->GetName().c_str());
      return FAILED;
    }
  } else {
    if (CompileMultipleOp(node_vec) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][CompileOp] Fail to compile op.");
      return FAILED;
    }
  }
  FE_TIMECOST_END(FEOpsKernelCompileOp, "FEOpsKernelInfoStore::CompileOp");
  return SUCCESS;
}

Status FEOpsKernelInfoStore::SetAtomicOpAttr(ge::OpDescPtr &op_desc, bool &atomic_node_flag) {
  std::vector<uint32_t> output_index;
  std::map<string, std::map<int64_t, int64_t>> sub_node_workspace_info;
  // only process when get output_index success
  std::vector<uint32_t> tmp_output_index;
  output_index = CompileGetAtomicOutput(op_desc);
  if (!output_index.empty()) {
    if (!ge::AttrUtils::SetListInt(op_desc, ge::ATOMIC_ATTR_OUTPUT_INDEX, output_index)) {
      std::map<std::string, std::string> error_key_map;
      error_key_map[EM_OP_NAME] = op_desc->GetName();
      error_key_map[EM_ERROR_MSG] = "Set output atomic info failed!";
      LogErrorMessage(EM_SET_ATTR_FAILED, error_key_map);
      REPORT_FE_ERROR("[SubGraphOpt][Compile][SetAtomOpAttr] Set op [%s] output atomic info to op_desc failed!",
                      op_desc->GetName().c_str());
      return FAILED;
    }
    atomic_node_flag = true;
  }

  // process with workspace info
  uint32_t atomic_flag = 0;
  if (ge::AttrUtils::GetInt(op_desc, TBE_OP_ATOMIC_WORKSPACE_FLAG, atomic_flag)) {
    if (atomic_flag == 1) {
      std::map<int64_t, int64_t> workspace_info;
      uint32_t workspace_index = 0;
      std::vector<int64_t> workspace_bytes_vec = op_desc->GetWorkspaceBytes();
      int64_t workspace_bytes = accumulate(workspace_bytes_vec.begin(), workspace_bytes_vec.end(), 0);
      workspace_info.insert(std::make_pair(workspace_index, workspace_bytes));
      sub_node_workspace_info.insert(std::make_pair(op_desc->GetName(), workspace_info));
      if (!op_desc->SetExtAttr(ge::EXT_ATTR_ATOMIC_WORKSPACE_INFO, sub_node_workspace_info)) {
        std::map<std::string, std::string> error_key_map;
        error_key_map[EM_OP_NAME] = op_desc->GetName();
        error_key_map[EM_ERROR_MSG] = "Set op workspace atomic info failed!";
        LogErrorMessage(EM_SET_ATTR_FAILED, error_key_map);
        REPORT_FE_ERROR("[SubGraphOpt][Compile][SetAtomOpAttr] Set op [%s] workspace atomic info failed!",
                        op_desc->GetName().c_str());
        return FAILED;
      }
      atomic_node_flag = true;
    }
  }

  FE_LOGD("Set op[name:%s,type:%s] workspace atomic info, atomic flag:%d, outputsize:%zu", op_desc->GetName().c_str(),
          op_desc->GetType().c_str(), atomic_flag, output_index.size());
  return SUCCESS;
}

Status FEOpsKernelInfoStore::CompileOpRun(vector<ge::NodePtr> &node_vec) {
  FE_TIMECOST_START(FEOpsKernelCompileOpRun);
  if (CompileOp(node_vec) != SUCCESS) {
    return FAILED;
  }
  FE_TIMECOST_END(FEOpsKernelCompileOpRun, "FEOpsKernelInfoStore::CompileOpRun");
  return SUCCESS;
}

Status FEOpsKernelInfoStore::GetOpImplyRealPath(std::string op_imply_relative_path, const OpImplType &op_impl_type,
                                                std::string &op_store_real_path, std::string &op_imply_real_path,
                                                const ge::NodePtr &node_ptr) {
  // get op_store Path prefix, for op imply path
  std::string op_imply_path_prefix;
  if (op_impl_type == EN_IMPL_NON_PERSISTENT_CUSTOM_TBE) {
    int32_t pos = op_store_real_path.rfind('/');
    if (pos < 0) {
      REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] The path of node %s dose not contain /.",
                      node_ptr->GetName().c_str());
      return FAILED;
    }
    op_imply_path_prefix = op_store_real_path.substr(0, pos);
  } else {
    op_imply_path_prefix = op_store_real_path;
  }

  // Check to see if it has refreshed op_imply_absolute_path
  if (op_imply_relative_path.find(op_imply_path_prefix) != string::npos) {
    FE_LOGD("Imply path of Op type[%s] has been refreshed, path is %s.", node_ptr->GetType().c_str(),
            op_imply_relative_path.c_str());
    op_imply_real_path = op_imply_relative_path;
    return SUCCESS;
  }

  // verify Relative Path, first char should not be '/'
  if (op_imply_relative_path.substr(0, 1) == "/") {
    REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Imply path of Op type[%s] is invalid, path is %s.",
                    node_ptr->GetType().c_str(), op_imply_relative_path.c_str());
    return FAILED;
  }

  std::string op_imply_absolute_path;
  if (op_imply_relative_path == "./") {
    op_imply_absolute_path = op_imply_path_prefix;
  } else {
    op_imply_absolute_path = op_imply_path_prefix + "/" + op_imply_relative_path;
  }

  op_imply_real_path = RealPath(op_imply_absolute_path);
  if (op_imply_real_path.empty()) {
    REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] The op_impl_file_path of node %s not exist in %s",
                    node_ptr->GetName().c_str(), op_imply_real_path.c_str());
    return FAILED;
  }
  return SUCCESS;
}

Status FEOpsKernelInfoStore::UpdateOpImplyPath(const ge::NodePtr &node_ptr, std::string &op_store_real_path,
                                               const OpImplType &op_impl_type,
                                               SubOpInfoStorePtr &sub_custom_ops_kernel_ptr) {
  FE_CHECK_NOTNULL(sub_custom_ops_kernel_ptr);
  std::string op_name = node_ptr->GetName();
  std::string op_type = node_ptr->GetType();

  // get op imply relative path from op_content
  OpContent op_content;
  Status status = sub_custom_ops_kernel_ptr->GetOpContentByOpType(op_type, op_content);
  if (status != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Op type[%s] not exist in op information library[%d].",
                    op_type.c_str(), op_impl_type);
    return FAILED;
  }

  OpKernelInfoConstructor op_kernel_info_constructor;
  std::string op_imply_relative_path;
  Status ret =
      op_kernel_info_constructor.GetStrFromOpContent(op_content, STR_IMP_PATH, STR_PATH, op_imply_relative_path);
  if (ret != SUCCESS || op_imply_relative_path.empty()) {
    if (op_impl_type == EN_IMPL_NON_PERSISTENT_CUSTOM_TBE) {
      REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Get op[%s]'s imply relative path failed.", op_name.c_str());
      return FAILED;
    } else {
      FE_LOGD("impl path of tbe-custom allow not exist.");
      return SUCCESS;
    }
  }

  std::string op_imply_real_path;
  if (GetOpImplyRealPath(op_imply_relative_path, op_impl_type, op_store_real_path, op_imply_real_path, node_ptr) !=
      SUCCESS) {
    FE_LOGD("Node %s get OpImplyRealPath failed.", op_name.c_str());
    return FAILED;
  }

  // set op imply absolute path to op_content
  OpKernelInfoPtr op_kernel_ptr = sub_custom_ops_kernel_ptr->GetOpKernelByOpType(op_type);
  if (op_impl_type != EN_IMPL_NON_PERSISTENT_CUSTOM_TBE && op_kernel_ptr != nullptr) {
    op_kernel_ptr->SetOpImpPath(op_imply_real_path);
  }

  map<string, string> map_temp;
  map_temp.emplace(std::make_pair(STR_PATH, op_imply_real_path));
  op_content.map_kernel_info_[STR_IMP_PATH] = map_temp;
  (void)sub_custom_ops_kernel_ptr->SetOpContent(op_content);
  return SUCCESS;
}

bool FEOpsKernelInfoStore::IsExistInTBECustom(const ge::NodePtr &node_ptr) {
  FEOpsStoreInfo op_store_info;
  if (SUCCESS != Configuration::Instance(engine_name_).GetOpStoreInfoByImplType(EN_IMPL_CUSTOM_TBE, op_store_info)) {
    FE_LOGI("Can not get tbe-custom op lib info by the imply_type.");
    return false;
  }

  OpKernelInfoPtr op_kernel_info_ptr =
      OpsKernelManager::Instance(engine_name_).GetOpKernelInfoByOpType(EN_IMPL_CUSTOM_TBE, node_ptr->GetType());
  if (op_kernel_info_ptr == nullptr) {
    FE_LOGI("Op[type=%s]: can not get the op kernel of %s.", node_ptr->GetType().c_str(), node_ptr->GetName().c_str());
    return false;
  }

  // updata tbe-custom op imply path if exist
  std::string op_store_real_path = RealPath(op_store_info.cfg_file_path);
  SubOpInfoStorePtr sub_def_custom_ops_kernel_ptr =
      OpsKernelManager::Instance(engine_name_).GetSubOpsKernelByStoreName(op_store_info.fe_ops_store_name);
  if (sub_def_custom_ops_kernel_ptr == nullptr) {
    FE_LOGI("There is no default custom op info library.");
    return false;
  }

  if (UpdateOpImplyPath(node_ptr, op_store_real_path, EN_IMPL_CUSTOM_TBE, sub_def_custom_ops_kernel_ptr) != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Update op[%s] imply path failed.",
                    node_ptr->GetName().c_str());
    return false;
  }
  return true;
}

Status FEOpsKernelInfoStore::GetDynamicCustomOpStoreInfoByNode(const ge::NodePtr &node_ptr,
                                                               vector<std::string> &json_files,
                                                               SubOpInfoStorePtr &sub_dyna_custom_ops_kernel_ptr) {
  FE_LOGD("GetDynamicCustomOpStoreInfo for node[%s, %s]", node_ptr->GetName().c_str(), node_ptr->GetType().c_str());
  std::string op_name = node_ptr->GetName();

  // if node's op_store_path attr not exist or value is empty, return
  std::string op_store_path;
  if ((!ge::AttrUtils::GetStr(node_ptr->GetOpDesc(), CUSTOM_OP_IMPL_CONFIG_PATH, op_store_path)) ||
      op_store_path.empty()) {
    if (IsExistInTBECustom(node_ptr) == false) {
      std::map<std::string, std::string> error_key_map;
      error_key_map[EM_OP_NAME] = node_ptr->GetName();
      error_key_map[EM_ERROR_MSG] = "The op information library path of node does not exist.";
      LogErrorMessage(EM_GET_FILEPATH_FAILED, error_key_map);
      REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] The op information library path of node %s not exist in %s",
                      op_name.c_str(), op_store_path.c_str());
      return FAILED;
    }
    return SUCCESS;
  }

  std::string op_store_real_path = RealPath(op_store_path);
  if (op_store_real_path.empty()) {
    std::map<std::string, std::string> error_key_map;
    error_key_map[EM_OP_NAME] = node_ptr->GetName();
    error_key_map[EM_ERROR_MSG] = "The op_store_file_path of node does not exist.";
    LogErrorMessage(EM_GET_FILEPATH_FAILED, error_key_map);
    REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] The op_store_file_path of node %s not exist in %s",
                    op_name.c_str(), op_store_path.c_str());
    return FAILED;
  }

  // don't need to load the same json file again
  bool has_been_load = false;
  for (auto &iter : json_files) {
    if (iter == op_store_real_path) {
      has_been_load = true;
    }
  }
  if (!has_been_load) {
    if (sub_dyna_custom_ops_kernel_ptr->LoadOpJsonFile(op_store_real_path) != SUCCESS) {
      std::map<std::string, std::string> error_key_map;
      error_key_map[EM_FILE] = op_store_real_path;
      error_key_map[EM_ERROR_MSG] = "Fail to load json file.";
      LogErrorMessage(EM_OPEN_FILE_FAILED, error_key_map);
      REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Fail to load json file[%s].", op_store_real_path.c_str());
      return FAILED;
    }
    json_files.push_back(op_store_real_path);
  }

  // updata op imply path
  if (UpdateOpImplyPath(node_ptr, op_store_real_path, EN_IMPL_NON_PERSISTENT_CUSTOM_TBE,
                        sub_dyna_custom_ops_kernel_ptr) != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Update op[%s] imply path failed.", op_name.c_str());
    return FAILED;
  }

  return SUCCESS;
}

Status FEOpsKernelInfoStore::GetDefFeOpsStoreInfo(FEOpsStoreInfo &fe_ops_store) {
  std::vector<FEOpsStoreInfo> fe_ops_store_info_vec = Configuration::Instance(engine_name_).GetOpsStoreInfo();

  for (auto ops_store_info : fe_ops_store_info_vec) {
    if (ops_store_info.op_impl_type == EN_IMPL_CUSTOM_TBE) {
      fe_ops_store = ops_store_info;
      return SUCCESS;
    }
  }

  return FAILED;
}

Status FEOpsKernelInfoStore::SetDynaCustomOpStoreToAllStore(FEOpsStoreInfo &fe_ops_store,
                                                            SubOpInfoStorePtr &sub_dyna_custom_ops_kernel_ptr) {
  Status status = sub_dyna_custom_ops_kernel_ptr->ConstructOpKernelInfo(engine_name_);
  if (status != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Fail to initialize non persistent custom sub ops kernel.");
    return FAILED;
  }
  status = OpsKernelManager::Instance(engine_name_).AddSubOpsKernel(sub_dyna_custom_ops_kernel_ptr);
  if (status != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Fail to add non persistent custom sub ops kernel into ops \
                    kernel manager.");
    return FAILED;
  }

  Configuration::Instance(engine_name_).SetOpsStoreInfo(fe_ops_store);
  SubOpsStorePtr sub_dyna_custom_ops_store_ptr = nullptr;
  FE_MAKE_SHARED(sub_dyna_custom_ops_store_ptr = std::make_shared<SubOpsStore>(op_store_adapter_manager_ptr_),
                 return FAILED);
  FE_CHECK(sub_dyna_custom_ops_store_ptr == nullptr,
           REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] subDynaCustomOpsStorePtr is nullptr."), return FAILED);

  sub_dyna_custom_ops_store_ptr->format_dtype_querier_ptr_ =
      std::make_shared<FormatDtypeQuerier>(op_store_adapter_manager_ptr_);
  FE_CHECK_NOTNULL(sub_dyna_custom_ops_store_ptr->format_dtype_querier_ptr_);

  sub_dyna_custom_ops_store_ptr->SetSubStoreInfo(fe_ops_store);
  sub_dyna_custom_ops_store_ptr->SetSubStoreType(fe_ops_store.fe_ops_store_name);
  map_all_sub_store_info_.emplace(std::make_pair(fe_ops_store.fe_ops_store_name, sub_dyna_custom_ops_store_ptr));

  return SUCCESS;
}

Status FEOpsKernelInfoStore::SetDynamicCustomOpStoreInfo(ge::ComputeGraph &graph) {
  FE_TIMECOST_START(SetDynamicCustomOpStoreInfo);
  FEOpsStoreInfo fe_ops_store = {NON_PERSISTENT_CUSTOM_PRIORITY,
                                 STR_NON_PERSISTENT_CUSTOM_TBE,
                                 EN_IMPL_NON_PERSISTENT_CUSTOM_TBE,
                                 "",
                                 "",
                                 true,
                                 true};
  FEOpsStoreInfo def_fe_ops_store;
  if (GetDefFeOpsStoreInfo(def_fe_ops_store) == SUCCESS) {
    fe_ops_store.cfg_file_path = def_fe_ops_store.cfg_file_path;
    fe_ops_store.op_impl_file_path = def_fe_ops_store.op_impl_file_path;
  }

  SubOpInfoStorePtr sub_dyna_custom_ops_kernel_ptr = nullptr;
  FE_MAKE_SHARED(sub_dyna_custom_ops_kernel_ptr = std::make_shared<SubOpInfoStore>(fe_ops_store),
                 return OP_STORE_MAKE_SHARED_FAILED);
  FE_CHECK(sub_dyna_custom_ops_kernel_ptr == nullptr,
           REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] subOpsKernelInfoStorePtr is nullptr."),
           return PARAM_INVALID);

  bool is_custom_op = false;
  bool has_dynamic_custom_op = false;
  vector<std::string> json_files;
  for (auto &node : graph.GetAllNodes()) {
    if ((ge::AttrUtils::GetBool(node->GetOpDesc(), NON_PERSISTENT_CUSTOM_OP_FLAG, is_custom_op)) && is_custom_op) {
      has_dynamic_custom_op = true;
      if (GetDynamicCustomOpStoreInfoByNode(node, json_files, sub_dyna_custom_ops_kernel_ptr) != SUCCESS) {
        REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Node %s get dynamic custom op_store info failed.",
                        node->GetName().c_str());
        return FAILED;
      }
    }
  }

  if (has_dynamic_custom_op) {
    if (SetDynaCustomOpStoreToAllStore(fe_ops_store, sub_dyna_custom_ops_kernel_ptr) != SUCCESS) {
      REPORT_FE_ERROR("[GraphOpt][SetDynmCustomOpStoreInfo] Set dynamic custom op_store info failed.");
      return FAILED;
    }
  }
  FE_TIMECOST_END(SetDynamicCustomOpStoreInfo,
                  "SetDynamicCustomOpStoreInfo during FEGraphOptimizer::OptimizeOriginalGraph");
  return SUCCESS;
}

bool FEOpsKernelInfoStore::OpCheckSupport(const ge::OpDescPtr &op_desc_ptr, FEOpsStoreInfo &ops_store,
                                          OpStoreAdapterPtr &op_store_adapter_ptr, OpKernelInfoPtr &op_kernel_info_ptr,
                                          UnSupportedReason &reason) const {
  std::ostringstream reason_oss;
  std::string unsupported_reason_in_op = te::NO_NEED_REASON;
  if (!op_store_adapter_ptr->CheckSupport(*(op_desc_ptr.get()), op_kernel_info_ptr, unsupported_reason_in_op)) {
    OpNotSupportedReasonID reason_id = EN_OPERATOR_NOT_SUPPORT;
    FE_LOGI("[ChkSpt][OpChk][Node %s, type %s] This op is not supported inside its implementation. Reason is %s.",
            op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str(), unsupported_reason_in_op.c_str());
    reason_oss << "Op " << op_desc_ptr->GetName() << " type " << op_desc_ptr->GetType().c_str()
               << " is not supported in the operator's implementation. Reason is: " << unsupported_reason_in_op << ".";
    uint64_t offset_num = ops_store.op_impl_type * NOT_SUPPORTED_REASON_OFFSET_BIT_NUM;
    reason.reason_id += (static_cast<uint64_t>(reason_id) << offset_num);
    reason.reason += reason_oss.str();
    return false;
  }

  return true;
}

bool FEOpsKernelInfoStore::CheckCustomOp(const ge::OpDescPtr &op_desc_ptr, FEOpsStoreInfo &ops_store,
                                         bool &is_custom_op) const {
  if ((ge::AttrUtils::GetBool(op_desc_ptr, NON_PERSISTENT_CUSTOM_OP_FLAG, is_custom_op)) && (is_custom_op)) {
    return ((ops_store.op_impl_type != EN_IMPL_CUSTOM_TBE) &&
            (ops_store.op_impl_type != EN_IMPL_NON_PERSISTENT_CUSTOM_TBE));
  } else {
    return (ops_store.op_impl_type == EN_IMPL_NON_PERSISTENT_CUSTOM_TBE);
  }
}

void FEOpsKernelInfoStore::SetGeOp(const ge::OpDescPtr &op_desc_ptr, const std::string &op_filename) const {
  if (op_filename == NULL_OP_FILE) {
    (void)ge::AttrUtils::SetBool(op_desc_ptr, IS_GE_OP, true);
  }
}

bool FEOpsKernelInfoStore::IsNeededCompile(ge::OpDescPtr &op_desc_ptr) {
  if (IsUnKnownShapeOp(*(op_desc_ptr.get()))) {
    bool is_support_unknown_shape = true;
    if (ge::AttrUtils::GetBool(op_desc_ptr, STR_SUPPORT_DYNAMIC_SHAPE, is_support_unknown_shape) &&
        !is_support_unknown_shape) {
      FE_LOGD("op[name:%s,type:%s] is not support dynamic shape, no need to do compile.",
              op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str());
      return false;
    }
  }

  std::vector<uint32_t> output_index = CompileGetAtomicOutput(op_desc_ptr);
  if (!output_index.empty()) {
    FE_LOGD("Op[name:%s,type:%s] has already been compiled, do not need compile.", op_desc_ptr->GetName().c_str(),
            op_desc_ptr->GetType().c_str());
    return false;
  }

  return true;
}
}  // namespace fe
